{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:02.534184Z",
     "start_time": "2024-05-10T05:19:02.529684Z"
    }
   },
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import time\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "# 用于文本清洗\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import nltk\n",
    "# 用于文本特征提取\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# 模型以及评估\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score\n",
    "\n",
    "# 定义数据所在路径\n",
    "TRAIN_PATH = r\"D:\\PycharmProjects\\tik-tok_-comment_-analysis\\nlp_date\\train1.csv\"\n",
    "TEST_PATH = r\"D:\\PycharmProjects\\tik-tok_-comment_-analysis\\nlp_date\\test_no_label.csv\""
   ],
   "execution_count": 516,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 数据分析与预处理"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "86c2b3b3cc6e87c9"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## pandas处理数据"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "33138a6658e9d304"
  },
  {
   "cell_type": "code",
   "source": [
    "# 利用pandas读入csv数据\n",
    "train_data = pd.read_csv(TRAIN_PATH)\n",
    "test_data = pd.read_csv(TEST_PATH)\n",
    "test_data['label'] = -1\n",
    "test_data.columns = train_data.columns  # 统一训练集和测试集的列标签\n",
    "\n",
    "print('shape of train: ', train_data.shape)\n",
    "print('shape of test: ', test_data.shape)\n",
    "\n",
    "# 查看数据的基本信息\n",
    "train_data.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:03.745876Z",
     "start_time": "2024-05-10T05:19:03.729459Z"
    }
   },
   "id": "e079e43ad1e010e",
   "execution_count": 517,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 查看数据的基本统计 - info\n",
    "train_data.info()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:05.013161Z",
     "start_time": "2024-05-10T05:19:05.007891Z"
    }
   },
   "id": "c8a461010ee1d4a3",
   "execution_count": 518,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 查看训练数据的类别分布，看是否存在类别不平衡问题\n",
    "train_data.groupby(['label']).size()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:05.811729Z",
     "start_time": "2024-05-10T05:19:05.801137Z"
    }
   },
   "id": "f4c398de68a6020b",
   "execution_count": 519,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 只取部分数据，作为测试使用；注，正式预测时，需要使用全量数据\n",
    "test_data = test_data[:2000].copy()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:06.682646Z",
     "start_time": "2024-05-10T05:19:06.673578Z"
    }
   },
   "id": "2cf5b4273ff54de2",
   "execution_count": 520,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 数据分析与清洗"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "4d878bbe2be03f15"
  },
  {
   "cell_type": "code",
   "source": [
    "# 打印一个文本内容进行查看\n",
    "train_data['txt'][0]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:07.558159Z",
     "start_time": "2024-05-10T05:19:07.551307Z"
    }
   },
   "id": "5986af9462548244",
   "execution_count": 521,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "#统计文本长度\n",
    "def Count_TxtLength(txts):\n",
    "    TxtLen = []\n",
    "    for txt in txts:\n",
    "        TxtLen.append(len(txt.split()))\n",
    "        \n",
    "    # 画图，横轴为文本长度，纵轴为数据累计占比\n",
    "    plt.hist(TxtLen, 800, density=True, stacked=True, cumulative=True)\n",
    "    plt.xlabel('Length of Text')\n",
    "    plt.ylabel('cumulative proportion')\n",
    "    plt.axhline(y=0.8, c='r')\n",
    "    plt.show()\n",
    "    return TxtLen"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:08.377699Z",
     "start_time": "2024-05-10T05:19:08.370143Z"
    }
   },
   "id": "8f33bef1d2742237",
   "execution_count": 522,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "%%time\n",
    "train_txtlen = Count_TxtLength(train_data['txt'])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:09.993143Z",
     "start_time": "2024-05-10T05:19:09.445818Z"
    }
   },
   "id": "c28530c2f9ef88fc",
   "execution_count": 523,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 特征工程"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "5b456a4befdf16a0"
  },
  {
   "cell_type": "code",
   "source": [
    "# 将训练集和测试集合并\n",
    "total_data = pd.concat([train_data, test_data], axis=0)\n",
    "print('训练集和测试集合并后的shape：', total_data.shape)\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:11.323192Z",
     "start_time": "2024-05-10T05:19:11.316688Z"
    }
   },
   "id": "6f13246806ace19c",
   "execution_count": 524,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:12.181691Z",
     "start_time": "2024-05-10T05:19:12.170861Z"
    }
   },
   "cell_type": "code",
   "source": "total_data",
   "id": "62628eb7e535b41a",
   "execution_count": 525,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1. 词袋表示 bag of words"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "14c0bbc753ce3928"
  },
  {
   "cell_type": "code",
   "source": [
    "# 初步构建词典，查看语料词典长度\n",
    "bow_vect = CountVectorizer(analyzer='word')\n",
    "bow_vect.fit(total_data['txt'])\n",
    "bow_vocab = bow_vect.get_feature_names_out() # 获取词典\n",
    "print('词典长度：', len(bow_vocab))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:13.856763Z",
     "start_time": "2024-05-10T05:19:13.835705Z"
    }
   },
   "id": "40bcc6c7fbcc9ff6",
   "execution_count": 526,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## TF-IDF表示"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "5db11aaef96bedd1"
  },
  {
   "cell_type": "code",
   "source": [
    "%%time\n",
    "# 使用tfidf权重作为文本的特征表示\n",
    "tfidf_vect = TfidfVectorizer(max_features=3000)\n",
    "tfidf_features = tfidf_vect.fit_transform(total_data['txt'])\n",
    "tfidf_vocab = tfidf_vect.get_feature_names_out()  # 获取词典\n",
    "print('词典长度：', len(tfidf_vocab))\n",
    "print('shape of tfidf representation:', tfidf_features.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:14.910491Z",
     "start_time": "2024-05-10T05:19:14.890160Z"
    }
   },
   "id": "3cf989da18c2cc90",
   "execution_count": 527,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 模型选择与优化"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "977dd00938827bd5"
  },
  {
   "cell_type": "code",
   "source": [
    "def Model_Optimization(model, params, train_x, train_y):\n",
    "    best_params = {}\n",
    "    for param in params:\n",
    "        print(\"optimize param：\", param)\n",
    "        cv = GridSearchCV(estimator=model, param_grid=param, \n",
    "                          scoring=\"f1_macro\", cv=3, n_jobs=-1, verbose=10)\n",
    "        cv.fit(train_x, train_y)\n",
    "        (key, value), = cv.best_params_.items()\n",
    "        best_params[key] = value\n",
    "    return cv.best_score_, best_params, cv.best_estimator_  # 返回最优评分、最优参数以及最优训练器"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:15.912308Z",
     "start_time": "2024-05-10T05:19:15.908157Z"
    }
   },
   "id": "e09aa9c6360a878a",
   "execution_count": 528,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 选择 bow 特征 或者 tfidf 特征构建训练数据\n",
    "train_x, train_y = tfidf_features[:3797], train_data['label'].values"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:16.722355Z",
     "start_time": "2024-05-10T05:19:16.719669Z"
    }
   },
   "id": "d4bcdd74305719c8",
   "execution_count": 529,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# lr模型的初始化与参数池\n",
    "lr = LogisticRegression(max_iter=1000)\n",
    "lr_params = [{'C':[0.001, 0.01, 0.1, 1, 5, 10]}]  # L2正则项的系数的倒数"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:17.405556Z",
     "start_time": "2024-05-10T05:19:17.397735Z"
    }
   },
   "id": "5e4a9ece67290121",
   "execution_count": 530,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "%%time\n",
    "lr_best_score, lr_best_params, lr_best = Model_Optimization(lr, lr_params, train_x, train_y)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:20.617063Z",
     "start_time": "2024-05-10T05:19:18.693081Z"
    }
   },
   "id": "c37457724bd687a8",
   "execution_count": 531,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "print(lr_best_score, '\\n', lr_best_params)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:21.669576Z",
     "start_time": "2024-05-10T05:19:21.659724Z"
    }
   },
   "id": "ada420450e6ab7fc",
   "execution_count": 532,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# xgb模型的初始化与参数池\n",
    "xgb = XGBClassifier()\n",
    "xgb_params = [{\"learning_rate\": [0.001, 0.01, 0.1, 1]}, \n",
    "              {\"n_estimators\": [20, 50, 100, 300]}, \n",
    "              {\"max_depth\": range(5,20,5)},   # 深度越大，拟合程度越高\n",
    "              {\"min_child_weight\": range(3,7,2)},   # 指建立模型所需要的最小样本数，调大可以控制过拟合\n",
    "              {\"gamma\": [i/10.0 for i in range(0,5,2)]},  # 给定了所需的最低loss func的值\n",
    "              {\"subsample\": [i/10.0 for i in range(6,10,2)]},  # 用于训练的子样本占整个样本集的比例，可防止过拟合\n",
    "              {\"reg_alpha\": [1e-5, 1e-2, 0.1, 1, 10]}]  # L1正则项的惩罚系数"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:22.383737Z",
     "start_time": "2024-05-10T05:19:22.370322Z"
    }
   },
   "id": "403d88f5245ec7c9",
   "execution_count": 533,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "%%time\n",
    "xgb_best_score, xgb_best_params, xgb_best = Model_Optimization(xgb, xgb_params, train_x, train_y)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:26.206746Z",
     "start_time": "2024-05-10T05:19:22.870338Z"
    }
   },
   "id": "91dc64bd1531475a",
   "execution_count": 534,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "print(xgb_best_score, '\\n', xgb_best_params)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:26.210446Z",
     "start_time": "2024-05-10T05:19:26.206746Z"
    }
   },
   "id": "2b071abb88b62ebc",
   "execution_count": 535,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 定义评估函数，并记录日志\n",
    "def Model_Evaluation(model, feature, train_x, train_y):\n",
    "    score = cross_val_score(model, train_x, train_y, scoring=\"f1_macro\", cv=3, verbose=10).mean()\n",
    "    with open(r\"D:\\PycharmProjects\\tik-tok_-comment_-analysis\\nlp_date\\test_no_label.csv\", \"a\") as f:\n",
    "        time_current = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\n",
    "        f.write(\"#\"*25 + time_current + \"#\"*25 + '\\n')   # 【时间 + ####】 标志新纪录开始\n",
    "        f.write(\"feature:\\t\" + feature + '\\n')\n",
    "        f.write(\"model:\\t\" + str(model) + '\\n')\n",
    "        f.write(\"score:\\t\" + str(score) + '\\n\\n')\n",
    "    return score"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:26.849122Z",
     "start_time": "2024-05-10T05:19:26.845668Z"
    }
   },
   "id": "612323a8613287df",
   "execution_count": 536,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "%%time\n",
    "# 记录lr、xgb结果\n",
    "lr_score = Model_Evaluation(lr_best, 'bow300', train_x, train_y)\n",
    "# xgb_score = Model_Evaluation(xgb_best, 'tfidf5000', train_x, train_y)\n",
    "print('lr_score: %.4f' % lr_score)\n",
    "# print('xgb_score: %.4f' % xgb_score)\n",
    "train_x.shape,len(train_y),tfidf_features"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:27.515394Z",
     "start_time": "2024-05-10T05:19:27.456208Z"
    }
   },
   "id": "17bdc473eaecb2f2",
   "execution_count": 537,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "def File_Submit(test_data, pred_y, model_name):\n",
    "    ids = test_data['ID'].values\n",
    "    submission = pd.DataFrame({\"ID\": ids, \"predicted_label\": pred_y})\n",
    "    submission.to_csv(r\"D:\\PycharmProjects\\tik-tok_-comment_-analysis\\nlp_date\\{}.csv\".format(model_name), index=False)\n",
    "    print('文件 {}.csv 已写出'.format(model_name))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:28.237578Z",
     "start_time": "2024-05-10T05:19:28.225248Z"
    }
   },
   "id": "c45a12d609f032db",
   "execution_count": 538,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "\n",
    "# test_x = bow_features[3797:]\n",
    "test_x = tfidf_features[3797:]\n",
    "\n",
    "#xgb_pred_y = xgb_best.predict(test_x)\n",
    "lr_pred_y = lr_best.predict(test_x)\n",
    "\n",
    "# File_Submit(test_data, xgb_pred_y, 'tfidf5000_xgb')\n",
    "File_Submit(test_data, lr_pred_y, 'bow300_lr')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-10T05:19:29.427112Z",
     "start_time": "2024-05-10T05:19:29.412695Z"
    }
   },
   "id": "e732bc6fa90d9b0c",
   "execution_count": 539,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-19T04:19:05.375617Z",
     "start_time": "2024-03-19T04:19:05.373526Z"
    }
   },
   "id": "7bb084dd01103e2d",
   "execution_count": 41,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "328d8bc6c40e86b5",
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
